{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "33bb4064",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-01T22:23:10.623889Z",
     "iopub.status.busy": "2021-10-01T22:23:10.549685Z",
     "iopub.status.idle": "2021-10-01T22:23:15.402690Z",
     "shell.execute_reply": "2021-10-01T22:23:15.402079Z",
     "shell.execute_reply.started": "2021-10-01T22:21:52.563589Z"
    },
    "papermill": {
     "duration": 4.869378,
     "end_time": "2021-10-01T22:23:15.402846",
     "exception": false,
     "start_time": "2021-10-01T22:23:10.533468",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-01 22:23:11.088050: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "import tensorflow_addons as tfa\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "pad_sequences = keras.preprocessing.sequence.pad_sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "58798487",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-01T22:23:15.431363Z",
     "iopub.status.busy": "2021-10-01T22:23:15.430816Z",
     "iopub.status.idle": "2021-10-01T22:23:23.367293Z",
     "shell.execute_reply": "2021-10-01T22:23:23.367709Z",
     "shell.execute_reply.started": "2021-10-01T22:21:52.577835Z"
    },
    "papermill": {
     "duration": 7.953133,
     "end_time": "2021-10-01T22:23:23.367860",
     "exception": false,
     "start_time": "2021-10-01T22:23:15.414727",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/imdb.npz\n",
      "17465344/17464789 [==============================] - 0s 0us/step\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<string>:6: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n",
      "/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/datasets/imdb.py:159: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n",
      "  x_train, y_train = np.array(xs[:idx]), np.array(labels[:idx])\n",
      "/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/datasets/imdb.py:160: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n",
      "  x_test, y_test = np.array(xs[idx:]), np.array(labels[idx:])\n"
     ]
    }
   ],
   "source": [
    "imdb = keras.datasets.imdb\n",
    "(train_data, train_labels), (test_data, test_labels) = imdb.load_data(num_words=10000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f455bf51",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-01T22:23:23.402679Z",
     "iopub.status.busy": "2021-10-01T22:23:23.402098Z",
     "iopub.status.idle": "2021-10-01T22:23:23.405774Z",
     "shell.execute_reply": "2021-10-01T22:23:23.406649Z",
     "shell.execute_reply.started": "2021-10-01T22:21:59.905413Z"
    },
    "papermill": {
     "duration": 0.02499,
     "end_time": "2021-10-01T22:23:23.406784",
     "exception": false,
     "start_time": "2021-10-01T22:23:23.381794",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_data, val_data, train_labels, val_labels = train_test_split(train_data, \n",
    "                                                                  train_labels, \n",
    "                                                                  test_size=0.30, \n",
    "                                                                  shuffle=True,\n",
    "                                                                  random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "99e24975",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-01T22:23:23.439816Z",
     "iopub.status.busy": "2021-10-01T22:23:23.438959Z",
     "iopub.status.idle": "2021-10-01T22:23:23.577004Z",
     "shell.execute_reply": "2021-10-01T22:23:23.576328Z",
     "shell.execute_reply.started": "2021-10-01T22:21:59.915091Z"
    },
    "id": "QVBYZDyfPdXl",
    "papermill": {
     "duration": 0.157055,
     "end_time": "2021-10-01T22:23:23.577202",
     "exception": false,
     "start_time": "2021-10-01T22:23:23.420147",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/imdb_word_index.json\n",
      "1646592/1641221 [==============================] - 0s 0us/step\n"
     ]
    }
   ],
   "source": [
    "# A dictionary mapping words to an integer index\n",
    "word_index = imdb.get_word_index()\n",
    "\n",
    "# The first indices are reserved\n",
    "word_index = {k:(v+3) for k,v in word_index.items()} \n",
    "word_index[\"<PAD>\"] = 0\n",
    "word_index[\"<START>\"] = 1\n",
    "word_index[\"<UNK>\"] = 2  # unknown\n",
    "word_index[\"<UNUSED>\"] = 3\n",
    "\n",
    "reverse_word_index = dict([(value, key) for (key, value) in word_index.items()])\n",
    "\n",
    "def decode_review(text):\n",
    "    return ' '.join([reverse_word_index.get(i, '?') for i in text])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "41d71b40",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-01T22:23:23.611155Z",
     "iopub.status.busy": "2021-10-01T22:23:23.610592Z",
     "iopub.status.idle": "2021-10-01T22:23:23.614103Z",
     "shell.execute_reply": "2021-10-01T22:23:23.614535Z",
     "shell.execute_reply.started": "2021-10-01T22:22:00.063848Z"
    },
    "papermill": {
     "duration": 0.022789,
     "end_time": "2021-10-01T22:23:23.614659",
     "exception": false,
     "start_time": "2021-10-01T22:23:23.591870",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import LeakyReLU\n",
    "from tensorflow.keras.layers import Activation\n",
    "from tensorflow.keras.optimizers import SGD, Adam\n",
    "\n",
    "from tensorflow.keras.wrappers.scikit_learn import KerasClassifier\n",
    "from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "58b4a563",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-01T22:23:23.652851Z",
     "iopub.status.busy": "2021-10-01T22:23:23.652349Z",
     "iopub.status.idle": "2021-10-01T22:23:25.776167Z",
     "shell.execute_reply": "2021-10-01T22:23:25.777364Z",
     "shell.execute_reply.started": "2021-10-01T22:22:00.071705Z"
    },
    "id": "C5vN2mLLPd28",
    "papermill": {
     "duration": 2.149247,
     "end_time": "2021-10-01T22:23:25.777606",
     "exception": false,
     "start_time": "2021-10-01T22:23:23.628359",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "pad_length = 256\n",
    "\n",
    "train_data = pad_sequences(train_data,\n",
    "                           value=word_index[\"<PAD>\"],\n",
    "                           padding='post',\n",
    "                           maxlen=pad_length)\n",
    "\n",
    "val_data = pad_sequences(val_data,\n",
    "                         value=word_index[\"<PAD>\"],\n",
    "                         padding='post',\n",
    "                         maxlen=pad_length)\n",
    "\n",
    "test_data = pad_sequences(test_data,\n",
    "                          value=word_index[\"<PAD>\"],\n",
    "                          padding='post',\n",
    "                          maxlen=pad_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "9be744bb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-01T22:23:25.834489Z",
     "iopub.status.busy": "2021-10-01T22:23:25.833591Z",
     "iopub.status.idle": "2021-10-01T22:23:25.838518Z",
     "shell.execute_reply": "2021-10-01T22:23:25.839715Z",
     "shell.execute_reply.started": "2021-10-01T22:22:01.412838Z"
    },
    "papermill": {
     "duration": 0.038694,
     "end_time": "2021-10-01T22:23:25.839902",
     "exception": false,
     "start_time": "2021-10-01T22:23:25.801208",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.layers import Dense, Dropout\n",
    "from tensorflow.keras.layers import Flatten, RepeatVector, dot, multiply, Permute, Lambda\n",
    "K = keras.backend\n",
    "\n",
    "def attention(layer):\n",
    "    # --- Attention is all you need --- #\n",
    "    _,_,units = layer.shape.as_list()\n",
    "    attention = Dense(1, activation='tanh')(layer)\n",
    "    attention = Flatten()(attention)\n",
    "    attention = Activation('softmax')(attention)\n",
    "    attention = RepeatVector(units)(attention)\n",
    "    attention = Permute([2, 1])(attention)\n",
    "    representation = multiply([layer, attention])\n",
    "    representation = Lambda(lambda x: K.sum(x, axis=-2), \n",
    "                            output_shape=(units,))(representation)\n",
    "    # ---------------------------------- #\n",
    "    return representation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "ca9322c3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-01T22:23:25.901790Z",
     "iopub.status.busy": "2021-10-01T22:23:25.900560Z",
     "iopub.status.idle": "2021-10-01T22:23:25.905272Z",
     "shell.execute_reply": "2021-10-01T22:23:25.906086Z",
     "shell.execute_reply.started": "2021-10-01T22:22:01.422973Z"
    },
    "papermill": {
     "duration": 0.043292,
     "end_time": "2021-10-01T22:23:25.906265",
     "exception": false,
     "start_time": "2021-10-01T22:23:25.862973",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def get_optimizer(option=0, learning_rate=0.001):\n",
    "    if option==0:\n",
    "        return tf.keras.optimizers.Adam(learning_rate)\n",
    "    elif option==1:\n",
    "        return tf.keras.optimizers.SGD(learning_rate, momentum=0.9, nesterov=True)\n",
    "    elif option==2:\n",
    "        return tfa.optimizers.RectifiedAdam(learning_rate)\n",
    "    elif option==3:\n",
    "        return tfa.optimizers.Lookahead(tf.optimizers.Adam(learning_rate), sync_period=3)\n",
    "    elif option==4:\n",
    "        return tfa.optimizers.SWA(tf.optimizers.Adam(learning_rate))\n",
    "    elif option==5:\n",
    "        return tfa.optimizers.SWA(tf.keras.optimizers.SGD(learning_rate, momentum=0.9, nesterov=True))\n",
    "    else:\n",
    "        return tf.keras.optimizers.Adam(learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "fdaffb64",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-01T22:23:25.963239Z",
     "iopub.status.busy": "2021-10-01T22:23:25.962449Z",
     "iopub.status.idle": "2021-10-01T22:23:25.981807Z",
     "shell.execute_reply": "2021-10-01T22:23:25.982500Z",
     "shell.execute_reply.started": "2021-10-01T22:22:01.446501Z"
    },
    "id": "1qiGmUv0dQCI",
    "papermill": {
     "duration": 0.052597,
     "end_time": "2021-10-01T22:23:25.982670",
     "exception": false,
     "start_time": "2021-10-01T22:23:25.930073",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "layers = keras.layers\n",
    "models = keras.models\n",
    "    \n",
    "def create_tunable_model(hp, vocab_size=10000, pad_length=256):\n",
    "\n",
    "    # Instantiate model params\n",
    "    embedding_size = hp.Int('embedding_size', min_value=8, max_value=512, step=8)\n",
    "    spatial_dropout = hp.Float('spatial_dropout', min_value=0, max_value=0.5, step=0.05)\n",
    "\n",
    "    conv_layers = hp.Int('conv_layers', min_value=1, max_value=5, step=1)\n",
    "    rnn_layers = hp.Int('rnn_layers', min_value=1, max_value=5, step=1)\n",
    "    dense_layers = hp.Int('dense_layers', min_value=1, max_value=3, step=1)\n",
    "\n",
    "    conv_filters = hp.Int('conv_filters', min_value=32, max_value=512, step=32)\n",
    "    conv_kernel = hp.Int('conv_kernel', min_value=1, max_value=8, step=1)\n",
    "\n",
    "    concat_dropout = hp.Float('concat_dropout', min_value=0, max_value=0.5, step=0.05)\n",
    "    dense_dropout = hp.Float('dense_dropout', min_value=0, max_value=0.5, step=0.05)\n",
    "\n",
    "    inputs = layers.Input(name='inputs',shape=[pad_length])\n",
    "    layer  = layers.Embedding(vocab_size, embedding_size, input_length=pad_length)(inputs)\n",
    "    layer  = layers.SpatialDropout1D(spatial_dropout)(layer)\n",
    "\n",
    "    for l in range(conv_layers):\n",
    "        if l==0:\n",
    "            conv = layers.Conv1D(filters=conv_filters, kernel_size=conv_kernel, \n",
    "                                 padding='valid', kernel_initializer='he_uniform')(layer)\n",
    "        else:\n",
    "            conv = layers.Conv1D(filters=conv_filters, kernel_size=conv_kernel, \n",
    "                                 padding='valid', kernel_initializer='he_uniform')(conv) \n",
    "\n",
    "    avg_pool_conv = layers.GlobalAveragePooling1D()(conv)\n",
    "    max_pool_conv = layers.GlobalMaxPooling1D()(conv)\n",
    "\n",
    "    representations = list()\n",
    "    for l in range(rnn_layers):\n",
    "        \n",
    "        use_bidirectional = hp.Choice(f'use_bidirectional_{l}', values=[0, 1])\n",
    "        use_lstm = hp.Choice(f'use_lstm_{l}', values=[0, 1])\n",
    "        units = hp.Int(f'units_{l}', min_value=8, max_value=512, step=8)\n",
    "\n",
    "        if use_lstm == 1:\n",
    "            rnl = layers.LSTM\n",
    "        else:\n",
    "            rnl = layers.GRU\n",
    "\n",
    "        if use_bidirectional==1:\n",
    "            layer = layers.Bidirectional(rnl(units, return_sequences=True))(layer)\n",
    "        else:\n",
    "            layer = rnl(units, return_sequences=True)(layer)\n",
    "\n",
    "        representations.append(attention(layer))\n",
    "\n",
    "    layer = layers.concatenate(representations + [avg_pool_conv, max_pool_conv])\n",
    "    layer = layers.Dropout(concat_dropout)(layer)\n",
    "\n",
    "    for l in range(dense_layers):\n",
    "        dense_units = hp.Int(f'dense_units_{l}', min_value=8, max_value=512, step=8)\n",
    "        layer = layers.Dense(dense_units)(layer)\n",
    "        layer  = layers.LeakyReLU()(layer)\n",
    "        layer = layers.Dropout(dense_dropout)(layer)\n",
    "\n",
    "    layer  = layers.Dense(1, name='out_layer')(layer)\n",
    "    outputs  = layers.Activation('sigmoid')(layer)\n",
    "\n",
    "    model  = models.Model(inputs=inputs, outputs=outputs)\n",
    "\n",
    "    hp_learning_rate = hp.Choice('learning_rate', values=[0.002, 0.001, 0.0005])\n",
    "    optimizer_type = hp.Choice('optimizer', values=list(range(6)))\n",
    "    optimizer = get_optimizer(option=optimizer_type, learning_rate=hp_learning_rate)\n",
    "    \n",
    "    model.compile(optimizer=optimizer,\n",
    "                  loss='binary_crossentropy',\n",
    "                  metrics=['acc'])\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "afba0940",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-01T22:23:26.033835Z",
     "iopub.status.busy": "2021-10-01T22:23:26.033016Z",
     "iopub.status.idle": "2021-10-01T22:23:26.089471Z",
     "shell.execute_reply": "2021-10-01T22:23:26.090112Z",
     "shell.execute_reply.started": "2021-10-01T22:22:01.466858Z"
    },
    "id": "v4AaBohkWErD",
    "papermill": {
     "duration": 0.084179,
     "end_time": "2021-10-01T22:23:26.090283",
     "exception": false,
     "start_time": "2021-10-01T22:23:26.006104",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import keras_tuner as kt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ac393d32",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-01T22:23:26.144616Z",
     "iopub.status.busy": "2021-10-01T22:23:26.143871Z",
     "iopub.status.idle": "2021-10-02T06:39:51.443315Z",
     "shell.execute_reply": "2021-10-02T06:39:51.443722Z"
    },
    "id": "3vznWJPcdQGt",
    "outputId": "57a3e20b-e211-4143-c553-d720c28040ac",
    "papermill": {
     "duration": 29785.329552,
     "end_time": "2021-10-02T06:39:51.443885",
     "exception": false,
     "start_time": "2021-10-01T22:23:26.114333",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trial 100 Complete [00h 03m 42s]\n",
      "val_acc: 0.876800000667572\n",
      "\n",
      "Best val_acc So Far: 0.8925333619117737\n",
      "Total elapsed time: 08h 16m 22s\n"
     ]
    }
   ],
   "source": [
    "tuner = kt.BayesianOptimization(hypermodel=create_tunable_model,\n",
    "                                objective='val_acc',\n",
    "                                max_trials=100,\n",
    "                                num_initial_points=3,\n",
    "                                directory='storage',\n",
    "                                project_name='imdb',\n",
    "                                seed=42)\n",
    "\n",
    "tuner.search(train_data, train_labels, \n",
    "             epochs=30,\n",
    "             batch_size=64, \n",
    "             validation_data=(val_data, val_labels),\n",
    "             shuffle=True,\n",
    "             verbose=2,\n",
    "             callbacks = [EarlyStopping('val_acc', patience=3, restore_best_weights=True)]\n",
    "             )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "55270e2c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-02T06:39:51.510338Z",
     "iopub.status.busy": "2021-10-02T06:39:51.509805Z",
     "iopub.status.idle": "2021-10-02T06:39:52.579690Z",
     "shell.execute_reply": "2021-10-02T06:39:52.579220Z"
    },
    "id": "vYf3mlBbVX25",
    "papermill": {
     "duration": 1.087822,
     "end_time": "2021-10-02T06:39:52.579824",
     "exception": false,
     "start_time": "2021-10-02T06:39:51.492002",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "best_hps = tuner.get_best_hyperparameters()[0]\n",
    "model = tuner.hypermodel.build(best_hps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "2324c102",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-02T06:39:52.614190Z",
     "iopub.status.busy": "2021-10-02T06:39:52.612562Z",
     "iopub.status.idle": "2021-10-02T06:39:52.616192Z",
     "shell.execute_reply": "2021-10-02T06:39:52.615736Z"
    },
    "papermill": {
     "duration": 0.022146,
     "end_time": "2021-10-02T06:39:52.616297",
     "exception": false,
     "start_time": "2021-10-02T06:39:52.594151",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'embedding_size': 264, 'spatial_dropout': 0.2, 'conv_layers': 1, 'rnn_layers': 2, 'dense_layers': 1, 'conv_filters': 192, 'conv_kernel': 3, 'concat_dropout': 0.4, 'dense_dropout': 0.15000000000000002, 'use_bidirectional_0': 0, 'use_lstm_0': 0, 'units_0': 464, 'dense_units_0': 384, 'learning_rate': 0.002, 'optimizer': 3, 'use_bidirectional_1': 0, 'use_lstm_1': 1, 'units_1': 512, 'dense_units_1': 136, 'dense_units_2': 360}\n"
     ]
    }
   ],
   "source": [
    "print(best_hps.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "a150d52f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-02T06:39:52.660556Z",
     "iopub.status.busy": "2021-10-02T06:39:52.650521Z",
     "iopub.status.idle": "2021-10-02T06:39:52.665019Z",
     "shell.execute_reply": "2021-10-02T06:39:52.664596Z"
    },
    "id": "upzApqVhdP-j",
    "papermill": {
     "duration": 0.034479,
     "end_time": "2021-10-02T06:39:52.665136",
     "exception": false,
     "start_time": "2021-10-02T06:39:52.630657",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "inputs (InputLayer)             [(None, 256)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding (Embedding)           (None, 256, 264)     2640000     inputs[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "spatial_dropout1d (SpatialDropo (None, 256, 264)     0           embedding[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "gru (GRU)                       (None, 256, 464)     1016160     spatial_dropout1d[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "lstm (LSTM)                     (None, 256, 512)     2000896     gru[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 256, 1)       465         gru[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 256, 1)       513         lstm[0][0]                       \n",
      "__________________________________________________________________________________________________\n",
      "flatten (Flatten)               (None, 256)          0           dense[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "flatten_1 (Flatten)             (None, 256)          0           dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "activation (Activation)         (None, 256)          0           flatten[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "activation_1 (Activation)       (None, 256)          0           flatten_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "repeat_vector (RepeatVector)    (None, 464, 256)     0           activation[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "repeat_vector_1 (RepeatVector)  (None, 512, 256)     0           activation_1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "permute (Permute)               (None, 256, 464)     0           repeat_vector[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "permute_1 (Permute)             (None, 256, 512)     0           repeat_vector_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "multiply (Multiply)             (None, 256, 464)     0           gru[0][0]                        \n",
      "                                                                 permute[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "multiply_1 (Multiply)           (None, 256, 512)     0           lstm[0][0]                       \n",
      "                                                                 permute_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv1d (Conv1D)                 (None, 254, 192)     152256      spatial_dropout1d[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "lambda (Lambda)                 (None, 464)          0           multiply[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_1 (Lambda)               (None, 512)          0           multiply_1[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling1d (Globa (None, 192)          0           conv1d[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d (GlobalMax (None, 192)          0           conv1d[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (None, 1360)         0           lambda[0][0]                     \n",
      "                                                                 lambda_1[0][0]                   \n",
      "                                                                 global_average_pooling1d[0][0]   \n",
      "                                                                 global_max_pooling1d[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "dropout (Dropout)               (None, 1360)         0           concatenate[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 384)          522624      dropout[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu (LeakyReLU)         (None, 384)          0           dense_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 384)          0           leaky_re_lu[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "out_layer (Dense)               (None, 1)            385         dropout_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_2 (Activation)       (None, 1)            0           out_layer[0][0]                  \n",
      "==================================================================================================\n",
      "Total params: 6,333,299\n",
      "Trainable params: 6,333,299\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "094a66cc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-02T06:39:52.708408Z",
     "iopub.status.busy": "2021-10-02T06:39:52.707506Z",
     "iopub.status.idle": "2021-10-02T06:39:52.785156Z",
     "shell.execute_reply": "2021-10-02T06:39:52.784664Z"
    },
    "papermill": {
     "duration": 0.105375,
     "end_time": "2021-10-02T06:39:52.785298",
     "exception": false,
     "start_time": "2021-10-02T06:39:52.679923",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "model.save(\"best_model.h5\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 29811.874925,
   "end_time": "2021-10-02T06:39:56.415559",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2021-10-01T22:23:04.540634",
   "version": "2.3.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
